{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "o3rvMRohg9AH",
    "outputId": "14dc137c-7e9e-4145-8c1a-07337685855c",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English tokens: ['hi', 'how', 'are', 'you']\n",
      "English IDs: [11662, 574, 263, 536]\n",
      "Chinese tokens: ['你', '好', '，', '这', '里', '是', '河', '北', '电', '视', '台', '。']\n",
      "Chinese IDs: [151, 354, 6, 912, 938, 553, 633, 233, 694, 845, 274, 5]\n",
      "Original text: 你好，这里是河北电视台。\n",
      "Tokenized IDs: [151, 354, 6, 912, 938, 553, 633, 233, 694, 845, 274, 5]\n",
      "Tokenized tokens: ['你', '好', '，', '这', '里', '是', '河', '北', '电', '视', '台', '。']\n",
      "Decoded text: 你 好 这 里 是 河 北 电 视 台\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import jieba\n",
    "from tokenizers import BertWordPieceTokenizer\n",
    "\n",
    "# 加载数据集\n",
    "data = pd.read_csv('lang.csv')\n",
    "\n",
    "# 去除多余的空格和特殊字符，并进行分词\n",
    "def clean_and_tokenize_chinese(text):\n",
    "    text = re.sub(r'\\s+', '', text)  # 去除多余的空格\n",
    "    return ' '.join(jieba.cut(text))\n",
    "\n",
    "data['Chinese'] = data['Chinese'].apply(lambda x: clean_and_tokenize_chinese(str(x)))\n",
    "data['English'] = data['English'].apply(lambda x: ' '.join(re.sub(r'\\s+', ' ', str(x)).strip().split()))\n",
    "\n",
    "# 将中文和英文数据分别保存为txt文件\n",
    "with open(\"chinese.txt\", 'w', encoding='utf-8') as f:\n",
    "    f.write(\"\\n\".join(data['Chinese']))\n",
    "\n",
    "with open(\"english.txt\", 'w', encoding='utf-8') as f:\n",
    "    f.write(\"\\n\".join(data['English']))\n",
    "\n",
    "# 创建BertWordPieceTokenizer标记器，包含常见的中文标点符号\n",
    "common_punctuations = ['。', '，', '！', '？', '：', '；', '（', '）', '“', '”', '‘', '’', '《', '》', '、']\n",
    "\n",
    "tokenizer_chinese = BertWordPieceTokenizer()\n",
    "tokenizer_chinese.train(files=[\"chinese.txt\"], vocab_size=60000, min_frequency=1, special_tokens=['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]'] + common_punctuations)\n",
    "\n",
    "tokenizer_english = BertWordPieceTokenizer()\n",
    "tokenizer_english.train(files=[\"english.txt\"], vocab_size=30000, min_frequency=2, special_tokens=['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]'])\n",
    "\n",
    "# 验证标记化结果\n",
    "encoded_en = tokenizer_english.encode(\"Hi how are you\")\n",
    "encoded_cn = tokenizer_chinese.encode(\"你好，这里是河北电视台。\")\n",
    "\n",
    "print(f\"English tokens: {encoded_en.tokens}\")\n",
    "print(f\"English IDs: {encoded_en.ids}\")\n",
    "\n",
    "print(f\"Chinese tokens: {encoded_cn.tokens}\")\n",
    "print(f\"Chinese IDs: {encoded_cn.ids}\")\n",
    "\n",
    "# 测试标记化结果是否符合预期\n",
    "sample_chinese_text = \"你好，这里是河北电视台。\"\n",
    "encoded = tokenizer_chinese.encode(sample_chinese_text)\n",
    "decoded = tokenizer_chinese.decode(encoded.ids)\n",
    "\n",
    "print(f\"Original text: {sample_chinese_text}\")\n",
    "print(f\"Tokenized IDs: {encoded.ids}\")\n",
    "print(f\"Tokenized tokens: {encoded.tokens}\")\n",
    "print(f\"Decoded text: {decoded}\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "from tokenizers import ByteLevelBPETokenizer\n",
    "from tqdm.notebook import tqdm\n",
    "from transformers import RobertaTokenizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.optim import AdamW\n",
    "from torchtext.data.metrics import bleu_score\n",
    "from transformers.optimization import get_linear_schedule_with_warmup\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "98aR7jb71qR5"
   },
   "source": [
    "# Position embedding and embedding "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "rkCydVCe1zSD"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import  torch.nn as nn\n",
    "import tensorflow as tf\n",
    "import  pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "\n",
    "\n",
    "\n",
    "class Embedding(nn.Module):\n",
    "    def __init__(self,vocab_size,embed_dim):\n",
    "        super(Embedding,self).__init__()\n",
    "        self.embedd=nn.Embedding(vocab_size,embed_dim)\n",
    "     \n",
    "\n",
    "    def forward(self,x):\n",
    "\n",
    "        out=self.embedd(x)\n",
    "        return out\n",
    "\n",
    "class PositionEmbedding(nn.Module):\n",
    "    def __init__(self,max_sen=5000,dim_data=512):\n",
    "        super(PositionEmbedding,self).__init__()\n",
    "        self.dim=dim_data\n",
    "\n",
    "        pos=torch.zeros(max_sen,dim_data)\n",
    "\n",
    "        for x in range(max_sen):\n",
    "            for y in range(0,dim_data,2):\n",
    "\n",
    "                pos[x,y]=math.sin(x/(1000**(2*y/self.dim)))\n",
    "                pos[x,y+1]=math.cos(x/(1000**((2*(y+1))/self.dim)))\n",
    "\n",
    "\n",
    "        pos=torch.unsqueeze(pos,0)\n",
    "        self.register_buffer('pe',pos)\n",
    "\n",
    "    def forward(self,x):\n",
    "\n",
    "        # The reason we increase the embedding values before addition is to make the positional encoding relatively smaller. This means the original meaning in the embedding vector won’t be lost when we add them together.\n",
    "\n",
    "        x=x*math.sqrt(self.dim)\n",
    "        sequnence_lenght=x.size(1) # Batch_size , sequence_length , dimensions\n",
    "  \n",
    "\n",
    "        output=x+torch.autograd.Variable(self.pe[:,:sequnence_lenght],requires_grad=False)\n",
    "\n",
    "        return output\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MjJ9_0hJ14x3"
   },
   "source": [
    "# Multi head attention block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "2bCMUJVT1-xg"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "import  torch.nn as nn\n",
    "import  torch\n",
    "import  torch.nn.functional as F\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import  pandas as pd\n",
    "\n",
    "class  MultiHeadattention(nn.Module):\n",
    "    def __init__(self,n_heads=8,dim=512):\n",
    "\n",
    "        super(MultiHeadattention,self).__init__()\n",
    "        self.size_of_matrix=int(dim/n_heads)\n",
    "        self.n_heads=n_heads\n",
    "        self.key=nn.Linear(dim,dim,bias=False)\n",
    "        self.query=nn.Linear(dim,dim,bias=False)\n",
    "        self.value=nn.Linear(dim,dim,bias=False)\n",
    "        self.out=nn.Linear(n_heads*self.size_of_matrix,dim)\n",
    "        self.droput=nn.Dropout(0.3)\n",
    "\n",
    "    def forward(self,key,value,query,mask=None):\n",
    "        batch_size=key.size(0)\n",
    "        seq_lenght=key.size(1)\n",
    "\n",
    "        batch_size_query=query.size(0)\n",
    "        batch_lenght_query=query.size(1)\n",
    "\n",
    "        key=self.key(key)\n",
    "        value=self.value(value)\n",
    "        query=self.query(query)\n",
    "        key=key.view(batch_size,seq_lenght,self.n_heads,self.size_of_matrix)\n",
    "\n",
    "        value =value.view(batch_size, seq_lenght, self.n_heads, self.size_of_matrix)\n",
    "\n",
    "        query = query.view( batch_size_query, batch_lenght_query, self.n_heads, self.size_of_matrix)\n",
    "\n",
    "        # key=self.key(key) #\n",
    "        # query=self.query(query)\n",
    "        # value=self.value(value)\n",
    "        key=key.transpose(1,2) # Transpose will convert it into batch_size , n_heads , seq_length , size_of_matrix\n",
    "        query=query.transpose(1,2)\n",
    "        value=value.transpose(1,2)\n",
    "\n",
    "        # convert the query matrix into the batch_size , n_heads, size_of_matrix , seq_lenght\n",
    "\n",
    "      \n",
    "        scalar_output=torch.matmul(query,key.transpose(-2,-1))\n",
    "\n",
    "        scalar_output_square_root=scalar_output/math.sqrt(self.size_of_matrix)\n",
    "      \n",
    "        if mask is not None:\n",
    "            mask=mask.unsqueeze(1)\n",
    "   \n",
    "           \n",
    "            scalar_output_square_root=scalar_output_square_root.masked_fill(mask==0,float('-inf'))  \n",
    "\n",
    "\n",
    "        output_from_softmax=F.softmax(scalar_output_square_root,dim=-1)\n",
    "\n",
    "\n",
    "        mutliplication_with_value=torch.matmul(self.droput(output_from_softmax),value)\n",
    "\n",
    "        concat=mutliplication_with_value.transpose(1,2).contiguous().view(batch_size, -1 ,self.n_heads*self.size_of_matrix)\n",
    "\n",
    "        outout_from_multihead=self.out(concat)\n",
    "\n",
    "        return outout_from_multihead , output_from_softmax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dfrqVn232GGD"
   },
   "source": [
    "# EncoderBlock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "id": "YO_ZkwneLyZX"
   },
   "outputs": [],
   "source": [
    "class Encoderlayer(nn.Module):\n",
    "    def __init__(self,dim,expansion_factor,num_heads=8):\n",
    "\n",
    "        super(Encoderlayer,self).__init__()\n",
    "        self.attn=MultiHeadattention(num_heads,dim)\n",
    "        self.droput=nn.Dropout(0.2) \n",
    "        self.Layernorm1=nn.LayerNorm(dim)\n",
    "        self.Layernorm2=nn.LayerNorm(dim)\n",
    "        self.feedwordnetwork=nn.Sequential(\n",
    "            nn.Linear( dim, expansion_factor*dim),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(dim*expansion_factor,dim)\n",
    "        )\n",
    "\n",
    "    def forward(self,x,mask=None):\n",
    "       \n",
    "        x1,att_prob=self.attn(x,x,x,mask)\n",
    "        x=self.Layernorm1(self.droput(x1)+x)\n",
    "        x1=self.feedwordnetwork(x)\n",
    "        x=self.Layernorm2(self.droput(x1)+x)\n",
    "        return x,att_prob\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "id": "E613KaJq2Lyg"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "class TransformerEncoderBlock(nn.Module):\n",
    "    def __init__(self,src_lenght,expansion_factor,dim=512,num_heads=8,max_sequence_lenght=5000):\n",
    "\n",
    "        super(TransformerEncoderBlock,self).__init__()\n",
    "\n",
    "        self.Embedding=Embedding(src_lenght,dim)\n",
    "        self.num_layers=6\n",
    "        self.positionEmbedding=PositionEmbedding(max_sen=max_sequence_lenght,dim_data=dim)\n",
    "        self.layers = nn.ModuleList([\n",
    "            Encoderlayer(dim,expansion_factor)\n",
    "            for _ in range(self.num_layers)\n",
    "        ])\n",
    "    \n",
    "    def forward(self,x,mask):\n",
    "        embeded_out = self.Embedding(x)\n",
    "        \n",
    "        x= self.positionEmbedding(embeded_out)\n",
    " \n",
    "\n",
    "        for layer in self.layers:\n",
    "            x,attn_prob=layer(x,mask)\n",
    "            \n",
    "            \n",
    "        return x,attn_prob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "h9pMUveR2TZA"
   },
   "source": [
    "# DecoderBlock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "id": "tKFnzsC0PrYH"
   },
   "outputs": [],
   "source": [
    "class DecoderLayer(nn.Module):\n",
    "    def __init__(self,expansion_factor,dim=512,num_heads=8):\n",
    "        super(DecoderLayer,self).__init__()\n",
    "        self.attn=MultiHeadattention(num_heads,dim)\n",
    "        self.attn2=MultiHeadattention(num_heads,dim)\n",
    "        self.n_layers=6\n",
    "        self.dropout_1=nn.Dropout(0.2)\n",
    "        self.Layer_1=nn.LayerNorm(dim)\n",
    "       \n",
    "        self.feedforward_network=nn.Sequential(\n",
    "\n",
    "            nn.Linear(dim,dim*expansion_factor),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(dim*expansion_factor , dim\n",
    "                      )\n",
    "        )\n",
    "\n",
    "    def forward(self,x,encoder_out,target_mask,src_mask):\n",
    "\n",
    "        x1,_=self.attn(x,x,x,target_mask)\n",
    "            # picking value from encoder outpout\n",
    "\n",
    "        x2=self.Layer_1(self.dropout_1(x1)+x)\n",
    "\n",
    "        x,attn_prob= self.attn2(encoder_out,encoder_out,x2,src_mask)\n",
    "\n",
    "        x = self.Layer_1(self.dropout_1(x2) + x)\n",
    "\n",
    "        x1=self.feedforward_network(x)\n",
    "        x=self.Layer_1(self.dropout_1(x1)+x)\n",
    "\n",
    "        return x,attn_prob\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "id": "7AJy1OtR2ZuX"
   },
   "outputs": [],
   "source": [
    "import  torch\n",
    "import torch.nn as nn\n",
    "\n",
    "\n",
    "class TransformerDecoderBlock(nn.Module):\n",
    "    def __init__(self,target_vocab_size,expansion_factor,dim=512,max_sequence_length=5000,num_heads=8):\n",
    "        super(TransformerDecoderBlock,self).__init__()\n",
    "        self.embedding=Embedding(target_vocab_size,dim)\n",
    "        self.position_embedding=PositionEmbedding(max_sen=max_sequence_length,dim_data=dim)\n",
    "        self.dropout_1=nn.Dropout(0.2)\n",
    "\n",
    "        self.layers = nn.ModuleList([\n",
    "            DecoderLayer(expansion_factor)\n",
    "            for _ in range(6)\n",
    "        ])\n",
    "\n",
    "    def forward(self,x,encoder_output,masked_target,src_mask):\n",
    "      \n",
    "        embedded_vector=self.embedding(x)\n",
    "        position=self.position_embedding(embedded_vector)\n",
    "        x=self.dropout_1(position)\n",
    "\n",
    "        for layer in self.layers:\n",
    "            x,attn_prob=layer(x,encoder_output,masked_target,src_mask)\n",
    "\n",
    "        return x,attn_prob     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5sQWZeX92fFn"
   },
   "source": [
    "# Transformer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "id": "CEfbzWNk2lQ3"
   },
   "outputs": [],
   "source": [
    "import  torch\n",
    "import torch.nn as nn\n",
    "\n",
    "\n",
    "class TransformerModel(nn.Module):\n",
    "    def __init__(self,vocab_size_source,vocab_size_target,dim,expansion_factor,DEVICE):\n",
    "        \n",
    "        super(TransformerModel, self).__init__()\n",
    "        \n",
    "        self.encoder=TransformerEncoderBlock(vocab_size_source,expansion_factor)\n",
    "       \n",
    "        self.decoder=TransformerDecoderBlock(vocab_size_target,expansion_factor)\n",
    "\n",
    "        self.logits=nn.Linear(dim,vocab_size_target)\n",
    "\n",
    "    def mask_target(self,target):\n",
    "        batch_size , seq_lenght = target.size(0),target.size(1)\n",
    "\n",
    "        masked_target = torch.tril(torch.ones(seq_lenght,seq_lenght)).expand(1,seq_lenght,seq_lenght)\n",
    "\n",
    "        # return masked_target.to(DEVICE,dtype=torch.long)\n",
    "        return  masked_target.to(DEVICE,dtype=torch.int64)\n",
    "\n",
    "    def pad_mask(self,x,idx=1) :\n",
    "\n",
    "        # This masking helps not to consider the indexes for training the padded values \n",
    "\n",
    "       mask=(x!=1).unsqueeze(-2)\n",
    "\n",
    "       return mask.to(DEVICE)\n",
    "\n",
    "        \n",
    "\n",
    "    def forward(self,x,y):\n",
    "        # source mask \n",
    "        src_mask=self.pad_mask(x).to(DEVICE)\n",
    "       \n",
    "        encoder_output,_=self.encoder(x,src_mask)\n",
    "      \n",
    "        masked_target= torch.bitwise_and(self.pad_mask(y) ,self.mask_target(y) )\n",
    "\n",
    "        decoder_output,attent_prob=self.decoder(y,encoder_output,masked_target,src_mask)\n",
    "       \n",
    "        output=self.logits(decoder_output)\n",
    "        probs_output=F.log_softmax(output,-1)\n",
    "\n",
    "        return probs_output\n",
    "        \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rHhhBPSY4W5D"
   },
   "source": [
    "# Creating English & Chinese dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "q6G7Ozmeg3cs"
   },
   "outputs": [],
   "source": [
    "class EnglishtoChineseDataset:\n",
    "    def __init__(self,english,Chinese,tokenizer_te,tokenizer_en,max_length=60):\n",
    "        self.english=english\n",
    "        self.Chinese=Chinese\n",
    "        self.max_len_en=max_length-1\n",
    "        self.max_len_te=max_length \n",
    "        self.tokenizer_en=tokenizer_en\n",
    "        self.tokenizer_te=tokenizer_te\n",
    "    def __len__(self):\n",
    "        return len(self.english)\n",
    "    def __getitem__(self,item):\n",
    "        english = self.tokenizer_en.encode(self.english[item]).ids\n",
    "        Chinese  = self.tokenizer_te.encode(self.Chinese[item]).ids\n",
    "\n",
    "        # english adding start ane end of the sentnece \n",
    "     \n",
    "        if len(english) > self.max_len_en-2:\n",
    "          english=[0]+english[:self.max_len_en-2]+[2]\n",
    "        else:\n",
    "          english=[0]+english+[2]\n",
    "\n",
    "        if len(english) < self.max_len_en:\n",
    "          english =english + [1]*(self.max_len_en-len(english))\n",
    "\n",
    "        if len(Chinese) > self.max_len_te-2:\n",
    "          Chinese=[0]+Chinese[:self.max_len_te-2]+[2]\n",
    "        else:\n",
    "          Chinese=[0]+Chinese+[2]\n",
    "\n",
    "        if len(Chinese) < self.max_len_te:\n",
    "          Chinese = Chinese + [1]*(self.max_len_te-len(Chinese))\n",
    "    \n",
    "        return {\n",
    "            \"src\":torch.tensor(english,dtype=torch.long),\n",
    "            \"targted\":torch.tensor(Chinese,dtype=torch.long)\n",
    "        }   \n",
    "      \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "QYN_OvZS1ETN"
   },
   "outputs": [],
   "source": [
    "def loss_fn(output,target):\n",
    "    loss_cal=nn.CrossEntropyLoss(ignore_index=1)\n",
    "    loss=loss_cal(output,target)\n",
    "\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "gSaoP74_SmhQ"
   },
   "outputs": [],
   "source": [
    "def covert_id_to_text(ids,tokenize,eos_indx):\n",
    "\n",
    "    if ids.dim()==1:\n",
    "      \n",
    "        output_tokens=[]\n",
    "        for token_ids in ids:\n",
    "            data_list=[]\n",
    "            data_list.append(token_ids)\n",
    "         \n",
    "          \n",
    "            if int(token_ids)==eos_indx:\n",
    "                break\n",
    "            else:\n",
    "              output_tokens.append(tokenize.decode(data_list).strip())       \n",
    "        return output_tokens        \n",
    "    elif ids.dim()==2 :\n",
    "        return [covert_id_to_text(ids[i, :], tokenize,eos_indx) for i in range(ids.size(0))]\n",
    "\n",
    "    raise RuntimeError(f' The dimesion should be 2 but we received {ids.dim()} ')    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "id": "CU_TWH_iRBbD"
   },
   "outputs": [],
   "source": [
    "actul=[]\n",
    "pred=[]\n",
    "import pickle \n",
    "def eval_fn(val_data, model):\n",
    "    model.eval()\n",
    "    total_loss=0\n",
    "    steps=0\n",
    "    hypotheses = []\n",
    "    references = []\n",
    "    with torch.no_grad():\n",
    "        tk = tqdm(val_data, total=len(val_data)) \n",
    "        for batch_size , data in enumerate(tk):\n",
    "            src=data['src']\n",
    "            target=data['targted']\n",
    "\n",
    "            src=src.to(DEVICE, dtype=torch.long)\n",
    "            target=target.to(DEVICE,dtype=torch.long)\n",
    "\n",
    "            output=model(src,target[:,:-1])\n",
    "            \n",
    "            \n",
    "            loss=loss_fn(output.view(-1,output.size(-1)),target[:,1:].contiguous().view(-1))\n",
    "\n",
    "            total_loss+=loss.item()\n",
    "\n",
    "            steps=+1\n",
    "\n",
    "            output = output.argmax(dim=-1)\n",
    "\n",
    "            target=target[:,1:]\n",
    "\n",
    "            # Calculating belu score \n",
    "            pred_tokens=covert_id_to_text(output,tokenizer_chinese,2)\n",
    "\n",
    "            actual_tokens=covert_id_to_text(target,tokenizer_chinese,2)\n",
    "\n",
    "\n",
    "            hypotheses += pred_tokens\n",
    "         \n",
    "       \n",
    "            references += [[token] for token in  actual_tokens]\n",
    "\n",
    "  \n",
    "    actul.append(actual_tokens)\n",
    "    pred.append(pred_tokens)\n",
    "    perplexity = np.log(total_loss / len(val_data))\n",
    "  \n",
    "    bleu_data = bleu_score(hypotheses, references)\n",
    "      \n",
    "\n",
    "    return bleu_data,perplexity\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "id": "gFnyitb5_TZ5"
   },
   "outputs": [],
   "source": [
    "def train_fn(train_data,model,optimizer,clip=1.0,scheduler=None):\n",
    "    model.train()\n",
    "    total_loss=0\n",
    "    step=0\n",
    "    tk = tqdm(train_data, total=len(train_data)) \n",
    "\n",
    "    for batch_size , train_data in enumerate(tk):\n",
    "        src=train_data['src']\n",
    "        target=train_data['targted']\n",
    "        src=src.to(DEVICE, dtype=torch.long)\n",
    "        target=target.to(DEVICE,dtype=torch.long)\n",
    "        optimizer.zero_grad()   \n",
    "        output=model(src,target[:,:-1])\n",
    "        loss=loss_fn(output.view(-1,output.size(-1)),target[:,1:].contiguous().view(-1)) # \n",
    "        total_loss+=loss.item()\n",
    "        steps=+1\n",
    "        output = output.argmax(dim=-1)\n",
    "        loss.backward()\n",
    "\n",
    "        nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
    "        optimizer.step()\n",
    "        perplexity = np.log(total_loss / len(train_data))\n",
    "\n",
    "\n",
    "\n",
    "    return output ,perplexity\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "id": "Foozht20q5m6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TransformerModel(\n",
       "  (encoder): TransformerEncoderBlock(\n",
       "    (Embedding): Embedding(\n",
       "      (embedd): Embedding(26272, 512)\n",
       "    )\n",
       "    (positionEmbedding): PositionEmbedding()\n",
       "    (layers): ModuleList(\n",
       "      (0-5): 6 x Encoderlayer(\n",
       "        (attn): MultiHeadattention(\n",
       "          (key): Linear(in_features=512, out_features=512, bias=False)\n",
       "          (query): Linear(in_features=512, out_features=512, bias=False)\n",
       "          (value): Linear(in_features=512, out_features=512, bias=False)\n",
       "          (out): Linear(in_features=512, out_features=512, bias=True)\n",
       "          (droput): Dropout(p=0.3, inplace=False)\n",
       "        )\n",
       "        (droput): Dropout(p=0.2, inplace=False)\n",
       "        (Layernorm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (Layernorm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (feedwordnetwork): Sequential(\n",
       "          (0): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Linear(in_features=1536, out_features=512, bias=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (decoder): TransformerDecoderBlock(\n",
       "    (embedding): Embedding(\n",
       "      (embedd): Embedding(7129, 512)\n",
       "    )\n",
       "    (position_embedding): PositionEmbedding()\n",
       "    (dropout_1): Dropout(p=0.2, inplace=False)\n",
       "    (layers): ModuleList(\n",
       "      (0-5): 6 x DecoderLayer(\n",
       "        (attn): MultiHeadattention(\n",
       "          (key): Linear(in_features=512, out_features=512, bias=False)\n",
       "          (query): Linear(in_features=512, out_features=512, bias=False)\n",
       "          (value): Linear(in_features=512, out_features=512, bias=False)\n",
       "          (out): Linear(in_features=512, out_features=512, bias=True)\n",
       "          (droput): Dropout(p=0.3, inplace=False)\n",
       "        )\n",
       "        (attn2): MultiHeadattention(\n",
       "          (key): Linear(in_features=512, out_features=512, bias=False)\n",
       "          (query): Linear(in_features=512, out_features=512, bias=False)\n",
       "          (value): Linear(in_features=512, out_features=512, bias=False)\n",
       "          (out): Linear(in_features=512, out_features=512, bias=True)\n",
       "          (droput): Dropout(p=0.3, inplace=False)\n",
       "        )\n",
       "        (dropout_1): Dropout(p=0.2, inplace=False)\n",
       "        (Layer_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (feedforward_network): Sequential(\n",
       "          (0): Linear(in_features=512, out_features=1536, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Linear(in_features=1536, out_features=512, bias=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (logits): Linear(in_features=512, out_features=7129, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# 定义模型\n",
    "INPUT_SIZE = tokenizer_english.get_vocab_size()\n",
    "OUTPUT_SIZE = tokenizer_chinese.get_vocab_size()\n",
    "DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model = TransformerModel(INPUT_SIZE, OUTPUT_SIZE, 512, 3, DEVICE)\n",
    "model.to(DEVICE)\n",
    "# model.load_state_dict(torch.load('model.pth',map_location=DEVICE))\n",
    "# # reloading the model \n",
    "\n",
    "# model.to(DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "id": "FK1vVGByyr_Z"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "\n",
    "def run():\n",
    "    df_train, df_valid = train_test_split(data, test_size=0.3, random_state=42)\n",
    "    df_train = df_train.reset_index(drop=True)\n",
    "    df_valid = df_valid.reset_index(drop=True)\n",
    "\n",
    "    train_data = EnglishtoChineseDataset(\n",
    "        english=df_train.English.values,\n",
    "        Chinese=df_train.Chinese.values,\n",
    "        tokenizer_te=tokenizer_chinese,\n",
    "        tokenizer_en=tokenizer_english\n",
    "    )\n",
    "\n",
    "    train_data_loader = torch.utils.data.DataLoader(\n",
    "        train_data,\n",
    "        batch_size=128\n",
    "    )\n",
    "\n",
    "    val_data = EnglishtoChineseDataset(\n",
    "        english=df_valid.English.values,\n",
    "        Chinese=df_valid.Chinese.values,\n",
    "        tokenizer_te=tokenizer_chinese,\n",
    "        tokenizer_en=tokenizer_english\n",
    "    )\n",
    "\n",
    "    validation_data_loader = torch.utils.data.DataLoader(\n",
    "        val_data,\n",
    "        batch_size=128\n",
    "    )\n",
    "\n",
    "    num_train_steps = int(len(train_data) / 128) * 15\n",
    "\n",
    "    param_optimizer = list(model.parameters())\n",
    "    optimizer = AdamW(param_optimizer, lr=1e-4)\n",
    "    scheduler = get_linear_schedule_with_warmup(\n",
    "        optimizer, \n",
    "        num_warmup_steps=0, \n",
    "        num_training_steps=num_train_steps\n",
    "    )\n",
    "\n",
    "    best_bleu4 = float('-inf')\n",
    "    es_patience = 3\n",
    "    patience = 0 \n",
    "\n",
    "    # 确保保存模型的目录存在\n",
    "    model_dir = 'Model'\n",
    "    if not os.path.exists(model_dir):\n",
    "        os.makedirs(model_dir)\n",
    "\n",
    "    model_path = os.path.join(model_dir, 'model.pth')\n",
    "\n",
    "    # 检查模型文件是否存在，如果存在则加载模型\n",
    "    if os.path.exists(model_path):\n",
    "        model.load_state_dict(torch.load(model_path, map_location=DEVICE))\n",
    "\n",
    "    for i in range(20):\n",
    "        print(\"Epoch {}/{}\".format(i+1, 20))\n",
    "\n",
    "        _ , train_prexlity = train_fn(train_data_loader, model, optimizer, clip=1.0, scheduler=scheduler)\n",
    "        blue_score, test_prexlity = eval_fn(validation_data_loader, model)\n",
    "\n",
    "        print(f'Epoch :{i+1} and  train prexlity {train_prexlity:.4f}')\n",
    "        print(f'Epoch :{i+1} and  test prexlity {test_prexlity:.4f} and belu score on validation is {blue_score}')\n",
    "\n",
    "        is_best = blue_score > best_bleu4\n",
    "        if is_best:\n",
    "            print(f'BLEU score improved ({best_bleu4:.4f} -> {blue_score:.4f}). Saving Model!')\n",
    "            best_bleu4 = blue_score\n",
    "            patience = 0\n",
    "            torch.save(model.state_dict(), model_path)\n",
    "        else:\n",
    "            patience += 1\n",
    "            print(f'Early stopping counter: {patience} out of {es_patience}')\n",
    "            if patience == es_patience:\n",
    "                print(f'Early stopping! Best BLEU4: {best_bleu4:.4f}')\n",
    "                break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "id": "q3lYwORR38o2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efa75300282a41ba9d8dedcf0a500ff5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed9dc51a76ce4eefbd8372007b3778aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :1 and  train prexlity 4.6590\n",
      "Epoch :1 and  test prexlity 1.0899 and belu score on validation is 0.16988809406757355\n",
      "BLEU score improved (-inf -> 0.1699). Saving Model!\n",
      "Epoch 2/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f57eddefaad42c4942effb99f9f4a0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "822e511f0ca04bd291db82532d4c4960",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :2 and  train prexlity 4.6265\n",
      "Epoch :2 and  test prexlity 1.0725 and belu score on validation is 0.1826229989528656\n",
      "BLEU score improved (0.1699 -> 0.1826). Saving Model!\n",
      "Epoch 3/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5999f9c30734d7e84eead4bd647cc40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e783f6a6690e4a97a3a36e6cdcb53ca7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :3 and  train prexlity 4.6083\n",
      "Epoch :3 and  test prexlity 1.0806 and belu score on validation is 0.1811583936214447\n",
      "Early stopping counter: 1 out of 3\n",
      "Epoch 4/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05e3f13f387c42d389279593b6053657",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "adb8923249fd410ba425192f7e59c18e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :4 and  train prexlity 4.5933\n",
      "Epoch :4 and  test prexlity 1.0854 and belu score on validation is 0.18255245685577393\n",
      "Early stopping counter: 2 out of 3\n",
      "Epoch 5/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2fcfdc1c254d4a42a6b9f61f452142e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ea69ac7e86e4b81aa1a678f77e54bcb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :5 and  train prexlity 4.5775\n",
      "Epoch :5 and  test prexlity 1.0653 and belu score on validation is 0.19484320282936096\n",
      "BLEU score improved (0.1826 -> 0.1948). Saving Model!\n",
      "Epoch 6/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "447da873eb3a4915ad87ff02e130484e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48b03dd8a80e49fa852aeb96e2edd79e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :6 and  train prexlity 4.5623\n",
      "Epoch :6 and  test prexlity 1.0613 and belu score on validation is 0.20117591321468353\n",
      "BLEU score improved (0.1948 -> 0.2012). Saving Model!\n",
      "Epoch 7/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89b6f7fca931453cb7a0660b80a34fd1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66c6ab1253ba43a18d31c858cf2979a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :7 and  train prexlity 4.5498\n",
      "Epoch :7 and  test prexlity 1.0479 and belu score on validation is 0.21267728507518768\n",
      "BLEU score improved (0.2012 -> 0.2127). Saving Model!\n",
      "Epoch 8/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e71d77b899934431b2f6435d59dfc0bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "946fbd272aaf4a679a0630ff06e83e73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :8 and  train prexlity 4.5307\n",
      "Epoch :8 and  test prexlity 1.0392 and belu score on validation is 0.21418717503547668\n",
      "BLEU score improved (0.2127 -> 0.2142). Saving Model!\n",
      "Epoch 9/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0ccf34779064cafb4519551f31efd48",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "155094255edc443ca93c31e7b28594cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :9 and  train prexlity 4.5094\n",
      "Epoch :9 and  test prexlity 1.0380 and belu score on validation is 0.22036662697792053\n",
      "BLEU score improved (0.2142 -> 0.2204). Saving Model!\n",
      "Epoch 10/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ef2aafbd0174eb68b1d16172c8f3783",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3441b900123457b96666a7b82e8a71d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :10 and  train prexlity 4.4883\n",
      "Epoch :10 and  test prexlity 1.0294 and belu score on validation is 0.22635617852210999\n",
      "BLEU score improved (0.2204 -> 0.2264). Saving Model!\n",
      "Epoch 11/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4bb76984f70543b79dfd35bfb21e26a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "289cc41dd3384296b6932677aec4430e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :11 and  train prexlity 4.4669\n",
      "Epoch :11 and  test prexlity 1.0274 and belu score on validation is 0.22777976095676422\n",
      "BLEU score improved (0.2264 -> 0.2278). Saving Model!\n",
      "Epoch 12/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7b95be2990649c9a54adab01d13a0e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b16ee8fab09444189f4124e8babc463",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :12 and  train prexlity 4.4483\n",
      "Epoch :12 and  test prexlity 1.0248 and belu score on validation is 0.23008868098258972\n",
      "BLEU score improved (0.2278 -> 0.2301). Saving Model!\n",
      "Epoch 13/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fb0d21cf5a84f0fa7bbfb3a519a2110",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f319002a86f4677ad2fafb1491d1784",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :13 and  train prexlity 4.4310\n",
      "Epoch :13 and  test prexlity 1.0160 and belu score on validation is 0.23901091516017914\n",
      "BLEU score improved (0.2301 -> 0.2390). Saving Model!\n",
      "Epoch 14/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2ad7c0bde1f4c549642774a0fa8c092",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a72b6b37327544d3bc68aec905f6cd3d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :14 and  train prexlity 4.4135\n",
      "Epoch :14 and  test prexlity 1.0222 and belu score on validation is 0.23722420632839203\n",
      "Early stopping counter: 1 out of 3\n",
      "Epoch 15/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de184e56921041948c01ec0a14c8dc9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e6529f8499d4f4dabbb3f8194a8d309",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :15 and  train prexlity 4.3973\n",
      "Epoch :15 and  test prexlity 1.0141 and belu score on validation is 0.24645328521728516\n",
      "BLEU score improved (0.2390 -> 0.2465). Saving Model!\n",
      "Epoch 16/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f38500cbdbb4dcd932dbac0bbcb1354",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "babc77a2427743fd831c032726ba4dbb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :16 and  train prexlity 4.3780\n",
      "Epoch :16 and  test prexlity 1.0140 and belu score on validation is 0.25095483660697937\n",
      "BLEU score improved (0.2465 -> 0.2510). Saving Model!\n",
      "Epoch 17/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0838b5a4eec14005a85e8b5d971c785a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17f7c73bc7ea4439b154cc1f1904c285",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :17 and  train prexlity 4.3661\n",
      "Epoch :17 and  test prexlity 1.0097 and belu score on validation is 0.2617328464984894\n",
      "BLEU score improved (0.2510 -> 0.2617). Saving Model!\n",
      "Epoch 18/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d89e32cb1f5d42bf8f54415e49ad0254",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7969c68541ab4b92ad446ddd1116585e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :18 and  train prexlity 4.3490\n",
      "Epoch :18 and  test prexlity 1.0067 and belu score on validation is 0.2672731578350067\n",
      "BLEU score improved (0.2617 -> 0.2673). Saving Model!\n",
      "Epoch 19/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2a22273006d4427b4bfe69dc9e2b3b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a648413d4e16450eb1b8ac7ac4012060",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :19 and  train prexlity 4.3346\n",
      "Epoch :19 and  test prexlity 1.0075 and belu score on validation is 0.2667500972747803\n",
      "Early stopping counter: 1 out of 3\n",
      "Epoch 20/20\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57985a457327425d8c106efa042085ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b7e74a3973a43ec826cce0c2af3ae9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch :20 and  train prexlity 4.3163\n",
      "Epoch :20 and  test prexlity 1.0014 and belu score on validation is 0.26767998933792114\n",
      "BLEU score improved (0.2673 -> 0.2677). Saving Model!\n"
     ]
    }
   ],
   "source": [
    "run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Transformer from scratch for Machine Translation - English to telugu.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "tf",
   "language": "python",
   "name": "tf"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
